#!/usr/bin/python
#coding=utf-8

import sys
import os
import rados
import threading
import re
import copy
import ConfigParser
import commands
import IPy
import logging
import logging.config

import global_list
from command import *

CONFPATH="/home/dongrui/liushuai/cephtest/mycluster"

class ReqMon:
    #self.h=链接集群的handle
    def __init__(self,handle,dict,reqid,logger):
        self.h=handle
        self.act=dict['act']
        self.val=dict['val']
        self.host=dict['host']
        self.reqid=int(reqid)
        self.logger=logger

    def ThreadDealer(self):
        resp={}
        resp['act']=str(self.act)
        resp['type']='mon'
        resp['status']='running'
        if self.act == 'getstats' or self.act == 'getstatsbyregex':
            resp['count']=0
        else:
            resp['host']=self.host

        #tname=threading.currentThread().getName()
        mtx=global_list.get_value('MUTEX')
        #加锁->获取结果列表->追加结果->更新列表->解锁
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

        #执行具体命令
        if self.act == 'getstats':
            ret=self.getstats()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'getstatsbyregex':
            ret=self.getstatsbyregex()
            if False == ret.has_key('err'):
                resp.update(ret)
                resp['status']='success'
            else:
                resp['status']='failed'
        elif self.act == 'start':
            ret=self.start()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'stop':
            ret=self.stop()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'restart':
            ret=self.restart()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'delete':
            ret=self.delete()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        elif self.act == 'add':
            ret=self.add()
            if ret == False:
                resp['status']='failed'
            else:
                resp['status']='success'
        else:
            pass
 
        #更新结果
        mtx=global_list.get_value('MUTEX')
        mtx.acquire()
        res=global_list.get_value('RESULTS')
        res[int(self.reqid)]=copy.deepcopy(resp)
        global_list.set_value('RESULTS',res)
        mtx.release()

    #辅助函数，得到主机列表
    def gethosts(self):
        respdict=dict()

        cmd="/usr/bin/ceph osd tree | /usr/bin/grep -w host | /usr/bin/awk '{print $4}'"
        ret,hosts=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("get ceph osd tree failed")
            respdict['err']='ceph osd tree failed'
            return respdict
        lines=hosts.split('\n')
        respdict['hosts']=lines
        
        return respdict

    #获取集群mon信息
    def getstats(self):
        respdict={}
        resplist=[]

        #查询mon状态
        mon_status = CephClusterCommand(self.h, prefix='mon_status', format='json')
        if mon_status.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" ceph mon_status failed, error is "+mon_status['err'])
            respdict['err']=mon_status['err']
            return respdict
        result=mon_status['result']

        #获取quorum状态
        quorum=result.get('quorum')
        out_quorum=result.get('outside_quorum')
        self.logger.info("reqid:"+str(self.reqid)+" mon status: quorum="+str(quorum)+", outside_quorum="+str(out_quorum))

        #获取所有mons信息
        mons=result.get('monmap').get('mons')
        for i in mons:
            d=dict()
            d['rank']=i.get('rank',-1)
            d['host']=i.get('name','None')
            d['name']="mon."+str(d['host'])
            d['public_addr']=i.get('addr','None')
            if d['rank'] in quorum:
                d['status']='running'
                d['quroum_status']='quorum'
            else:
                d['status']='stopped'
            if d['rank'] in out_quorum:
                d['quroum_status']='out of quroum'
            resplist.append(d)

        respdict['mons']=resplist
        respdict['count']=len(mons)
        self.logger.info("reqid:"+str(self.reqid)+" success")
        
        return respdict

    #模糊查询集群mon信息
    def getstatsbyregex(self):
        respdict={}
        resplist=[]

        #查询mon状态
        mon_status = CephClusterCommand(self.h, prefix='mon_status', format='json')
        if mon_status.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" ceph mon_status failed, error is "+mon_status['err'])
            respdict['err']=mon_status['err']
            return respdict
        result=mon_status['result']

        #获取quorum状态
        quorum=result.get('quorum')
        out_quorum=result.get('outside_quorum')
        self.logger.info("reqid:"+str(self.reqid)+" mon status: quorum="+str(quorum)+", outside_quorum="+str(out_quorum))

        #获取所有mons信息
        templist=result.get('monmap').get('mons')
        mons=[ x for x in templist if None != re.search(self.val,"mon."+str(x['name']))] 
        self.logger.info("reqid:"+str(self.reqid)+" mon regex lists: "+str(mons))
        for i in mons:
            d=dict()
            d['rank']=i.get('rank',-1)
            d['host']=i.get('name','None')
            d['name']="mon."+str(d['host'])
            d['public_addr']=i.get('addr','None')
            if d['rank'] in quorum:
                d['status']='running'
                d['quroum_status']='quorum'
            else:
                d['status']='stopped'
            if d['rank'] in out_quorum:
                d['quroum_status']='out of quroum'
            resplist.append(d)

        respdict['mons']=resplist
        respdict['count']=len(mons)
        self.logger.info("reqid:"+str(self.reqid)+" success")
        return respdict

    #启动某个mon服务
    def start(self):
        #远程启动mon
        cmd="/usr/bin/systemctl -H "+str(self.host)+" start ceph-mon@"+str(self.host)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" start ceph-mon@"+str(self.host)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(self.host)+" reset-failed ceph-mon@"+str(self.host)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(self.host)+" start ceph-mon@"+str(self.host)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" start ceph-mon@"+str(self.host)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" start ceph-mon@"+str(self.host)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found")
                return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" start ceph-mon@"+str(self.host)+" success")

        #检查服务是否启动成功
        cmd="/usr/bin/systemctl -H "+str(self.host)+" is-active ceph-mon@"+str(self.host)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res == 'active':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is active now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #停止某个mon服务
    def stop(self):
        #远程停止mon
        cmd="/usr/bin/systemctl -H "+str(self.host)+" stop ceph-mon@"+str(self.host)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" stop ceph-mon@"+str(self.host)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(self.host)+" reset-failed ceph-mon@"+str(self.host)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(self.host)+" stop ceph-mon@"+str(self.host)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" stop ceph-mon@"+str(self.host)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" stop ceph-mon@"+str(self.host)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found")
                return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" stop ceph-mon@"+str(self.host)+" success")

        #检查服务是否停止成功
        cmd="/usr/bin/systemctl -H "+str(self.host)+" is-active ceph-mon@"+str(self.host)+".service"
        #is-active的返回值是3，而不能判断是否为0
        ret,res=commands.getstatusoutput(cmd)
        if res == 'inactive':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is inactive now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #重启某个mon服务
    def restart(self):
        #远程重启mon
        cmd="/usr/bin/systemctl -H "+str(self.host)+" restart ceph-mon@"+str(self.host)+".service"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" restart ceph-mon@"+str(self.host)+" failed, output: "+output)
            if output.find("systemctl reset-failed") != -1:
                self.logger.info("reqid:"+str(self.reqid)+" find reset-failed contents")
                cmd="/usr/bin/systemctl -H "+str(self.host)+" reset-failed ceph-mon@"+str(self.host)+".service"
                ret=os.system(cmd)
                if ret != 0:
                    self.logger.error("reqid:"+str(self.reqid)+" reset-failed failed")
                    return False
                else:
                    self.logger.info("reqid:"+str(self.reqid)+" reset-failed success")
                    cmd="/usr/bin/systemctl -H "+str(self.host)+" restart ceph-mon@"+str(self.host)+".service"
                    ret=os.system(cmd)
                    if ret != 0:
                        self.logger.error("reqid:"+str(self.reqid)+" restart ceph-mon@"+str(self.host)+" failed 2nd")
                        return False
                    self.logger.info("reqid:"+str(self.reqid)+" restart ceph-mon@"+str(self.host)+" 2nd success")
            else:
                self.logger.error("reqid:"+str(self.reqid)+" no reset-failed found")
                return False
        else:
            self.logger.info("reqid:"+str(self.reqid)+" restart ceph-mon@"+str(self.host)+" success")

        #检查服务是否重启成功
        cmd="/usr/bin/systemctl -H "+str(self.host)+" is-active ceph-mon@"+str(self.host)+".service"
        ret,res=commands.getstatusoutput(cmd)
        if res == 'active':
            self.logger.info("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is active now")
            return True
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-mon@"+str(self.host)+" is-active failed, ret="+str(ret)+", res="+res)
            return False

    #删除一个mon节点
    def delete(self):
        #首先查找mon的ip
        mon_dump = CephClusterCommand(self.h, prefix='mon dump', format='json')
        if mon_dump.has_key('err') == True:
            self.logger.error("reqid:"+str(self.reqid)+" mon dump failed, error is "+mon_dump['err'])
            return False
        result=mon_dump['result']
        mons=result.get('mons','None')
        if mons == 'None':
            self.logger.error("reqid:"+str(self.reqid)+" mons is None")
            return False

        #根据rank值查找对应的mon的host
        addr=''
        for i in mons:
            if i.get('name','None') == str(self.host):
                addr_port=i.get('addr','None')
                addr=addr_port.split(':')[0]
                self.logger.info("reqid:"+str(self.reqid)+" delete mon host is "+str(self.host)+",ip is "+str(addr))
            else:
                continue
        if addr == '':
            self.logger.error("reqid:"+str(self.reqid)+" hostname not found")
            return False

        #ceph-deploy destroy
        cmd="/usr/bin/ceph-deploy mon destroy "+str(self.host)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-deploy mon destroy failed")
            return False

        #停止mon服务进程
        cmd="/usr/bin/systemctl -H "+str(self.host)+" stop ceph-mon@"+str(self.host)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" stop ceph-mon@"+str(self.host)+" failed")
            return False

        #拷贝一个配置文件的副本
        cmd="/usr/bin/cp -p /etc/ceph/ceph.conf /etc/ceph/ceph.conf.bak"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf->ceph.conf.bak failed")
            return False
        
        #修改ceph.conf.bak
        config=ConfigParser.ConfigParser()
        with open("/etc/ceph/ceph.conf.bak","r") as cfgfile:
            config.readfp(cfgfile)
            #修改mon_initial_members，删除要移除的host
            mon_initial_members=config.get("global","mon_initial_members")
            members_list=mon_initial_members.replace(' ','').split(',')
            self.logger.debug("reqid:"+str(self.reqid)+" read option members_list is "+str(members_list))
            members_list.remove(str(self.host))
            members_str=','.join(members_list)
            self.logger.debug("reqid:"+str(self.reqid)+" write option members_list is "+str(members_str))

            #修改mon_host，删除要移除的ip
            mon_host=config.get("global","mon_host")
            host_list=mon_host.replace(' ','').split(',')
            self.logger.debug("reqid:"+str(self.reqid)+" read option host_list is "+str(host_list))
            for i in host_list:
                if i.split(':')[0] == addr:
                    host_list.remove(i)
            host_str=','.join(host_list)
            self.logger.debug("reqid:"+str(self.reqid)+" write option host_list is "+str(host_str))
            
            #设置mon_initial_members和mon_host
            config.set("global","mon_initial_members",members_str)
            config.set("global","mon_host",host_str)
            config.write(open("/etc/ceph/ceph.conf.bak","w"))

        #push到各个节点
        hosts=self.gethosts()
        if True == hosts.has_key('err'):
            self.logger.error("reqid:"+str(self.reqid)+" get hosts failed")
            return False
      
        #将配置文件拷贝到各个主机，但并不覆盖原始文件
        for i in hosts['hosts']:
            cmd="/usr/bin/scp /etc/ceph/ceph.conf.bak "+str(i)+":/etc/ceph/ceph.conf.bak"
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf.bak->remote hosts failed")
                return False

        #在各个节点上mv文件
        for i in hosts['hosts']:
            cmd="/usr/bin/ssh "+str(i)+" /usr/bin/mv -f /etc/ceph/ceph.conf.bak /etc/ceph/ceph.conf"
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" mv ceph.conf.bak on remote hosts failed")
                return False

        #本节点也要mv
        cmd="/usr/bin/mv -f /etc/ceph/ceph.conf.bak /etc/ceph/ceph.conf"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" mv ceph.conf.bak on localhost failed")
            return False

        self.logger.info("reqid:"+str(self.reqid)+" delete ceph-mon@"+str(self.host)+" success") 
        return True

    #添加一个mon节点
    def add(self):
        #从配置文件中获取public_network
        config=ConfigParser.ConfigParser()
        with open("/etc/ceph/ceph.conf","r") as cfgfile:
            config.readfp(cfgfile)
            #获取public_network
            public_net=config.get("global","public_network")
        self.logger.info("reqid:"+str(self.reqid)+" read option public_network is "+str(public_net))

        #获取主机所有IP
        cmd="/usr/bin/ssh "+str(self.host)+" /usr/sbin/ip addr | /usr/bin/grep -w inet | /usr/bin/awk '{print $2} | /usr/bin/awk -F'/' '{print $1}'"
        ret,output=commands.getstatusoutput(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" get host "+str(self.host)+" ipaddr failed")
            return False

        #获取public网段ip地址
        monIP=''
        iplist=output.split('\n')
        for i in iplist:
            if i in IPy.IP(public_net):
                self.logger.info("reqid:"+str(self.reqid)+" get host "+str(self.host)+" public ipaddr "+str(i))
                monIP=i
                break
        if monIP == '':
            self.logger.error("reqid:"+str(self.reqid)+" get host "+str(self.host)+" public ipaddr failed")
            return False

        #ceph-deploy add
        #进入xx目录，拷贝conf，执行添加mon命令
        os.chdir(CONFPATH)
        cmd="/usr/bin/cp -p /etc/ceph/ceph.conf "+str(CONFPATH)
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf to deploy dir failed")
            return False

        cmd="/usr/bin/ceph-deploy mon add "+str(self.host)
        ret=os.system(cmd)
        if ret == 0:
            self.logger.info("reqid:"+str(self.reqid)+" ceph-deploy mon add "+str(self.host)+" success")
        else:
            self.logger.error("reqid:"+str(self.reqid)+" ceph-deploy mon add "+str(self.host)+" failed")
            return False

        #拷贝一个配置文件的副本
        cmd="/usr/bin/cp -p /etc/ceph/ceph.conf /etc/ceph/ceph.conf.bak"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf->ceph.conf.bak failed")
            return False

        #修改ceph.conf.bak
        #config=ConfigParser.ConfigParser()
        with open("/etc/ceph/ceph.conf.bak","r") as cfgfile:
            config.readfp(cfgfile)
            #修改mon_initial_members
            mon_initial_members=config.get("global","mon_initial_members")
            members_list=mon_initial_members.replace(' ','').split(',')
            self.logger.info("reqid:"+str(self.reqid)+" read option members_list is "+str(members_list))
            members_list.append(str(self.host))
            members_str=','.join(members_list)
            self.logger.info("reqid:"+str(self.reqid)+" write option members_list is "+str(members_str))

            #修改mon_host
            mon_host=config.get("global","mon_host")
            host_list=mon_host.replace(' ','').split(',')
            self.logger.info("reqid:"+str(self.reqid)+" read option host_list is "+str(host_list))
            host_list.append(monIP)
            host_str=','.join(host_list)
            self.logger.info("reqid:"+str(self.reqid)+" write option host_list is "+str(host_str))
            
            #设置mon_initial_members和mon_host
            config.set("global","mon_initial_members",members_str)
            config.set("global","mon_host",host_str)
            config.write(open("/etc/ceph/ceph.conf.bak","w"))

        #push到各个节点
        hosts=self.gethosts()
        if True == hosts.has_key('err'):
            self.logger.error("reqid:"+str(self.reqid)+" get hosts failed")
            return False
      
        #将配置文件拷贝到各个主机，但并不覆盖原始文件
        for i in hosts['hosts']:
            cmd="/usr/bin/scp /etc/ceph/ceph.conf.bak "+str(i)+":/etc/ceph/ceph.conf.bak"
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" cp ceph.conf.bak->remote hosts failed")
                return False
        
        #在各个节点上mv文件
        for i in hosts['hosts']:
            cmd="/usr/bin/ssh "+str(i)+" /usr/bin/mv -f /etc/ceph/ceph.conf.bak /etc/ceph/ceph.conf"
            ret=os.system(cmd)
            if ret != 0:
                self.logger.error("reqid:"+str(self.reqid)+" mv ceph.conf.bak on remote hosts failed")
                return False

        #本节点也要mv
        cmd="/usr/bin/mv -f /etc/ceph/ceph.conf.bak /etc/ceph/ceph.conf"
        ret=os.system(cmd)
        if ret != 0:
            self.logger.error("reqid:"+str(self.reqid)+" mv ceph.conf.bak on localhost failed")
            return False

        self.logger.info("reqid:"+str(self.reqid)+" add ceph-mon@"+str(self.host)+" success")
        return True

    #析构
    def __del__(self):
        pass
